import requests
from lxml import etree
import re

class DouBan(object):

    def __init__(self):
        # self.url = "https://movie.douban.com/top250?start=25&filter="
        self.headers = {
            'User-Agent':"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36"
        }

    def get_data(self):
        resp = requests.get(self.url, headers=self.headers)
        resp_str = resp.content.decode()
        return resp_str

    def parse_data(self, data):
        html = etree.HTML(data)
        div_list = html.xpath("//div[@class='article']//div[@class='item']")
        data_list = []
        for div in div_list:
            item = {}
            item["title"] = div.xpath("//span[1][@class='title']/text()")[div_list.index(div)]
            item["rating_num"] = div.xpath("//span[@class='rating_num']/text()")[div_list.index(div)]
            item['comment'] = div.xpath("//div[@class='star']/span[4]/text()")[div_list.index(div)]
            data_list.append(item)
        return data_list

    def save_data(self, data_list):
        for data in data_list:
            # print(len(data_list))
            print(data)

    def run(self):
        for i in range(10):
            self.url = "https://movie.douban.com/top250?start=page&filter="
            # self.url = "https://movie.douban.com/top250?start=page&filter="
            # print(self.url)
            self.url = re.sub(r'page', str(int(25*i)), self.url)
            # print(self.url)
            data = self.get_data()
            data_list = self.parse_data(data)
            self.save_data(data_list)

if __name__ == "__main__":
    douban = DouBan()
    douban.run()



